# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

from pymongo import MongoClient

from giligili_crawler import items, settings


class GiligiliCrawlerPipeline(object):

    def __init__(self):
        self.client = None
        self.db = None
        self.collections = None

    def process_item(self, item, spider):
        if type(item) == items.GiliArticleItem:
            self.process_article(item, spider)
        elif type(item) == items.GiliArticleDetailItem:
            self.process_detail(item, spider)
        elif type(item) == items.VideoUrlItem:
            self.process_video(item, spider)
        return item

    def open_spider(self, spider):
        self.client = MongoClient(settings.MONGODB_HOST, settings.MONGODB_PORT)
        self.db = self.client[settings.MONGODB_DB_NAME]
        self.collections = self.db[settings.MONGODB_COLLECTION_NAME]

    def close_spider(self, spider):
        self.client.close()

    def process_article(self, item, spider):
        data = self.collections.find_one({'article_id': item['article_id']})

        if data:
            data.update(item)
            self.collections.update({'article_id': data['article_id']}, data)
        else:
            self.collections.insert_one(dict(item))

    def process_detail(self, item, spider):
        data = self.collections.find_one({'article_id': item['article_id']})

        if data:
            data.update(item)
            self.collections.update({'article_id': data['article_id']}, data)
        else:
            self.collections.insert_one(dict(item))

    def process_video(self, item, spider):
        data = self.collections.find_one({"article_id": item['article_id']})
        if data:
            if 'urls' not in data:
                data['urls'] = list()
            data['urls'] = list(set(data['urls'] + item['urls']))
            self.collections.update({'_id': data['_id']}, data)
        else:
            self.collections.insert_one(dict(item))
